#from celery import Celery
from CrawlerQueue import CrawlerQueue
from datetime import datetime,timedelta
from pymongo import MongoClient,errors
from bs4 import BeautifulSoup

import requests
import urllib.parse
import urllib.request
import urllib.robotparser
import math
import db as mysql
import datetime,json,random,socket,csv,time, re,queue,sys
from models import TbTieba,TbTiebaUser,TbUser,TbUserEducation,TbUserWork
from sqlalchemy import Column, String, create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql import text
# import crawler

#app=Celery('tasks', backend='redis://localhost:6379/0', broker='redis://localhost:6379/0')

client=MongoClient('192.168.0.124',27017)
db=client.cache

clientwriter=MongoClient('192.168.0.103',27017)
dbwriter=clientwriter.cache

def getproxy():
    client=MongoClient('192.168.0.124',27017)
    db=client.cache
    a=random.randint(1, db.proxy.find({'status':2}).count()-1)
    ipport=db.proxy.find({'status':2}).limit(1).skip(a-1)
    #print(ipport[0])
    return {'http':'http://%s:%s/'%(ipport[0]['_id'],ipport[0]['port'])}
    #return None
def gethtml(url):
    try:
        proxy=None
        html=requests.get(url,proxies=proxy,timeout=10)
        return html.text
    except Exception as identifier:
        return False  


def crawltiebauser():
    cq=CrawlerQueue(db.tiebaurl)
    tbuser=cq.pop()
    
    while tbuser:
        html=gethtml(tbuser['_id'])
        if html:
            try:
                bs=BeautifulSoup(html,'html.parser')
                namelist = bs.find_all(attrs={'class':'drl_item_name'})
                levellist = bs.find_all(attrs={'class':'drl_item_title'})
                jiangyanlist=bs.find_all(attrs={'class':'drl_item_exp'})
                x=-1
                session=mysql.DBSession()
                tbuserlist=[]
                for l in namelist:
                    
                    x=x+1
                    # dbwriter.tieba_user.insert({
                    #     'tieba':tbuser['tieba'],
                    #     'name':l.text,
                    #     'level':int(levellist[x].next.attrs['class'][0][5:]),
                    #     'jingyan':int(jiangyanlist[x].text),
                    #     'status':0
                    # })
                    print(l.text)
                    try:
                       
                        tieba_user=TbTiebaUser()
                        tieba_user.exp=int(jiangyanlist[x].text)
                        tieba_user.level=int(levellist[x].next.attrs['class'][0][5:])
                        tieba_user.tiebaname=tbuser['tieba']
                        tieba_user.username=l.text                
                        tbuserlist.append(tieba_user)
                    except Exception as identifier:
                        pass
                    session.add_all(tbuserlist)
                    session.commit()
                    session.close()

                cq.delqueue(tbuser['_id'])
                print(str(x)+",complete")
            except Exception as identifier:
                print(identifier)
                cq.remove(tbuser['_id'])
        else:
            cq.remove(tbuser['_id'])
        tbuser=cq.pop()

if __name__=='__main__':
    crawltiebauser()